package sis.ra.founded;

import java.io.File;
import java.util.ArrayList;

import sis.ra.utility.Utils;


public class crawlingFounded {

	public static String crawlInfoBoxPartCompany(String wikidoc)
	{
		String infobox="";
		if (wikidoc.length()>10) 
		{	String[] lines=wikidoc.split("\n");
			for (int i=0;i<lines.length;i++)
			{
//				System.out.println(lines[i]);
				if (lines[i].toLowerCase().contains("{{infobox")&&lines[i].toLowerCase().contains("company"))
				{
					while (!lines[i].equals("}}")&&!lines[i].trim().startsWith("}}")&&!lines[i].equals("|}}")&&!lines[i].endsWith("|}}"))
					{
//						System.out.println(lines[i]);
						if (lines[i].startsWith("|"))
							infobox=infobox+"| "+lines[i].substring(1).trim()+" ";
						else
							infobox=infobox+lines[i].trim()+" ";
						i=i+1;
					}
					break;
				}
			}		
		}
		return infobox;
	}
	
	public static void crawlInfoBox()
	{
		String companynamefile="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\yahoo company\\company.txt";
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\nasdaq100\\";
		String [] companys=Utils.readFile(companynamefile).split("\n");
		
		for (int i=0;i<companys.length;i++)
		{
			String companyname=companys[i];
			System.out.println(companyname);
			String wikipage="http://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=API|"+companyname.replace(" ", "%20")+"&rvprop=content&format=xml";
			File fl=new File (path+companyname+".infobox");
			if (fl.exists()) continue;
			if (fl.toString().contains("Microsoft")) continue; //semi-protected
			String wikicontent=Utils.fetchContentfrURL(wikipage);
			String infoboxcompany=crawlInfoBoxPartCompany(wikicontent);
	//		Utils.writeFile(path+companyname+".infobox", infoboxcompany);			
		}
	}
	
	 /**
     * This method checks if a String contains only numbers
     */
    public static boolean containsNumbers(String str) {
        
        //It can't contain only numbers if it's null or empty...
        if (str == null || str.length() == 0)
            return false;
        
        for (int i = 0; i < str.length(); i++) {

            //If we find a non-digit character we return false.
            if (Character.isDigit(str.charAt(i)))
                return true;
        }
        
        return false;
    }
    
	public static String links(String locations)
	{
		String links=locations;
//		System.out.println(locations);
		if (containsNumbers(locations))
		{
			links="";
		}
		if (locations.contains("[["))
		{
			links="";	
			String [] lines=locations.split("\\[\\[");
			for (int i=1;i<lines.length;i++)
			{
				if (lines[i].contains("]]"))
				{
					
					String com=lines[i].substring(0,lines[i].indexOf("]]"));
	//				System.out.println(com.trim());
					if (containsNumbers(com)) continue;
					if (com.trim().contains("|"))
					{
						String [] t=com.trim().split("\\|");
						for (int j=0;j<t.length;j++)
						{
							links=links+t[j]+"\t";
						}
					}else
					{
						links=links+com.trim()+"\t";
					}
				}				
			}
		}
		return links.trim();
	}
	
	public static String extractFoundationFrmInfobox(String line)
	{
		String foundation="";
		String [] lines=line.split("\\| ");
		for (int i=0;i<lines.length;i++)
		{
			if (lines[i].trim().startsWith("location")||lines[i].trim().startsWith("foundation"))
			{
	//			System.out.println(lines[i]);
			//[[Zune]]<ref>[http://www.zune.net/en-US/ zune.net]</ref>
				String locations=lines[i].trim().replace("&lt;br&gt;", "").replace("&lt;br /&gt;", "").trim();
		//		System.out.println(locations);
				if (locations.split("=").length>1)
				{
					locations=locations.split("=")[1];
		//			System.out.println(locations);
					String links=links(locations.trim());
					foundation=foundation+links+"\t";
				}
			}
		}
		return foundation;
	}
	
	public static void main(String[] args) {
	//	crawlInfoBox();
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\nasdaq100\\";
		File fl=new File (path);
		ArrayList companylocation = new ArrayList ();
		File [] files=fl.listFiles();
		int countNoInfoBox=0;
		int countFounder=0;
		int countFoundation=0;
		for (int i=0;i<files.length;i++)
		{
			File file=files[i];
			String companyname=file.toString().replace(path, "").replace(".infobox", "");
	//		System.out.println(companyname);
			
			if (companyname.contains(" "))
				companyname="\""+companyname+"\"";
			String content=Utils.readFile(file.toString());
			if (content.length()<1) 
			{
				countNoInfoBox=countNoInfoBox+1;
				continue;
			}
			content=content.substring(content.indexOf("{{Infobox"));
			content=content.replace("&lt;", "<").replace("&gt;", ">");
			if (content.contains("founder")) countFounder=countFounder+1;
			if (content.contains("foundation")) 
			{
				countFoundation=countFoundation+1;
				String links=extractFoundationFrmInfobox(content);
				links=links.trim();
				
				String [] locations=links.split("\t");
				for (int j=0;j<locations.length;j++)
				{
					String locname=locations[j];
					if (locations[j].contains(" "))
						locname="\""+locations[j]+"\"";
					if (locname.contains("<!--"))
						continue;
					if (!companylocation.contains(companyname+" "+locname))
					{
						companylocation.add(companyname+" "+locname);
					}
				}
			}
	//		System.out.println(content);
		}
		
		for (int i=0;i<companylocation.size();i++)
		{
			System.out.println(companylocation.get(i));
		}
		System.out.println("No InfoBox: "+countNoInfoBox);
		System.out.println("No Founder: "+countFounder);
		System.out.println("No Foundation: "+countFoundation);
		System.out.println("Location number "+companylocation.size());
	}

}
